import urllib2
import re
import os
import sys
import multiprocessing as mp
import socket

def geturl(path):
	try:
		s = urllib2.urlopen(path).read()
	except Exception, e:
		print e
		return False, e
	return True, s

def run(l, r):
	for x in xrange(l, r):
		xprefix = x / 1000

		status, s = geturl("http://www.wenku8.com/novel/{}/{}/".format(xprefix, x))
		if not status:
			continue

		p = re.compile('\d+.htm')
		match = p.findall(s)
		web_name = 'website/novel{}.htm'.format(x)
		print web_name
		if not os.path.exists(web_name):
			with open(web_name, 'w') as f:
				f.write(s)
		for y in match:
			status, t = geturl("http://www.wenku8.com/novel/{}/{}/{}".format(xprefix, x, y))
			if not status:
				continue

			inner_web_path = 'website/{}'.format(y)
			if not os.path.exists(inner_web_path):
				with open(inner_web_path, 'w') as f:
					f.write(t)
			print inner_web_path
			pp = re.compile('\d+.jpg')
			m = pp.findall(t)
			for z in m:
				url = "http://pic.wkcdn.com/pictures/{}/{}/{}/{}".format(xprefix, x, y.split('.')[0], z)
				if os.path.exists('website/pics/{}'.format(z)):
					continue
				status, pic = geturl(url)
				if not status:
					continue
				pic_path = 'website/pics/{}'.format(z)
				print pic_path
				with open(pic_path, 'wb') as f:
					f.write(pic)

def start():
	if not os.path.exists('website'):
		os.mkdir('website')
	if not os.path.exists('website/pics'):
		os.mkdir('website/pics')
	socket.setdefaulttimeout(60)
	plist = []
	for x in xrange(0, 2020, 1):
		mp_kwargs = dict(l=x, r=x+1)
		p = mp.Process(target=run, kwargs=mp_kwargs)
		p.start()
		#plist.append(p)
	#for x in plist:
	#	x.join()

		
if __name__ == '__main__':
	start()
